# -*- coding: utf-8 -*-

import unicodedata

from Products.ZCTextIndex.ISplitter import ISplitter
from Products.ZCTextIndex.PipelineFactory import element_factory
from Products.CMFPlone.utils import classImplements

from core import bigrams, kananormalize
from config import *

def to_unicode(s):
    """ str to unicode, replace to stop word. """
    if isinstance(s, str):
        try:
            s = s.decode(enc)
        except (UnicodeDecodeError, TypeError):
            pass
    t = u"".join(t for t in s if t not in STOP_WORD)
    return t

class BigramSplitter(object):
    """
    """
    meta_type = 'BigramSplitter'
    __implements__ = ISplitter

    def process(self, lst, glob=0):
        ulst = (to_unicode(t) for t in lst)
        return [t.encode(enc, 'replace') for t in bigrams(ulst, glob)]

    def processGlob(self, lst):
        return self.process(lst, 1)

classImplements(BigramSplitter, BigramSplitter.__implements__)

try:
    element_factory.registerFactory('Word Splitter',
        'Bigram Splitter', BigramSplitter)
except ValueError:
    # In case the splitter is already registered, ValueError is raised
    pass


class BigramCaseNormalizer(object):

    def process(self, lst):
        result = []
        for s in lst:
            try:
                if not isinstance(s, unicode):
                    s = unicode(s, enc)
            except (UnicodeDecodeError, TypeError):
                result.append(s.lower())
            else:
                normaed = unicodedata.normalize('NFKC', s)
                kana_normaed = kananormalize(normaed)
                result.append(kana_normaed.lower().encode(enc, 'replace'))
        return result

try:
    element_factory.registerFactory('Case Normalizer',
        'Bigram Case Normalizer', BigramCaseNormalizer)
except ValueError:
    # In case the normalizer is already registered, ValueError is raised
    pass
